"use strict"

/**
 * 网络爬虫接口模块
 *  
 */


var cheerio = require("cheerio");
var request = require("request");
var BufferHelper = require("bufferhelper");
var iconv = require("iconv-lite");
var response_fn = function (body, encode, cb, err) {
    body = body || "";
    if (body == "") {
        console.log("当前页返回空，请确认!");
        cb("", err);
    } else {
        var data = iconv.decode(body, encode);
        cb(data);
    }
};

var headers = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/52.0.2743.116 Chrome/52.0.2743.116 Safari/537.36"
};

var download_get = function (timeout, url, encode, cb) {
    //request.setTimeout(10000); // http request timeout set 10 seconds
    request.get({
        headers: headers,
        url: url,
        encoding: null,
        timeout: timeout
    }, function (err, httpResponse, body) {
        if (err) {
            body = "";
        }
        response_fn(body, encode, cb, err);
    });
};

var download_post = function (timeout, url, encode, cb, post_data) {
    request.post({
        headers: headers,
        url: url,
        encoding: null,
        timeout: timeout
    }, post_data || {}, function (err, httpResponse, body) {
        response_fn(body, encode, cb);
    });
};


exports = module.exports = {
    version: "1.0.0",
    name: "crawler spider support",
    // timeout: 10000,
    get: function (url, cb) {
        download_get(this.timeout, url, this.encode, cb);
    },
    post: function (url, post_data, cb) { // 暂时只是stub，没有支持细节
        download_post(this.timeout, url, this.encode, cb, post_data);
    },
    encode: "utf8"
}; 